from urllib import parse

import facade
from bs4 import BeautifulSoup
from xjlibrary.our_file_dir import BaseDir

curPath = BaseDir.get_file_dir_absolute(__file__)
configfile = BaseDir.get_new_path(curPath, "db.ini")
logger = facade.get_streamlogger()
mysqlutils = facade.MysqlUtiles(configfile, "db", logger=logger)
gClassName = 'OTHER'
ListSqls = list()
nStart = 1
nCount = 0


# 获取wanId
def GetFileName(sPath):
    global ListSqls
    print('Start!!!')

    soup = BeautifulSoup(open(sPath, 'r', encoding='utf-8'), "lxml")
    # 获取类名为pinglun_table的table标签
    table = soup.find('table', class_='pinglun_table')
    trs = table.find_all('tr')
    print('trs:' + str(len(trs)))
    # 便利tr 但除去第一个
    for tr in trs[1:]:
        # print(tr)
        tds = tr.find_all('td')
        # 获取第一个td下的a标签的href属性
        href = tds[0].find('a').get('href')
        # 获取多个字符串并去除空格等
        catalog = ''.join(tds[1].stripped_strings)
        timespan = ''.join(tds[2].stripped_strings)
        pubdate = ''.join(tds[3].stripped_strings)
        # 获取url的get参数，parse.parse_qs将参数转换成字典，键为str，value为list
        vid = parse.parse_qs(parse.urlparse(href).query)['VideoID'][0]

        sql = "INSERT IGNORE INTO article(vid, catalog, timespan, pubdate) VALUES('{vid}', '{catalog}', '{timespan}', '{pubdate}') ".format(
            vid=vid,
            catalog=catalog,
            timespan=timespan,
            pubdate=pubdate)
        ListSqls.append(sql)
    # 插入数据库
    InsertIntoDbFromList()


# 插入数据库
def InsertIntoDbFromList():
    global ListSqls
    mysqlutils.ExeSqlListToDB(ListSqls)
    ListSqls = list()


# ----------------------------------------------逻辑部分-----------------------
# ----------------------------------------------录入数据库wanID-----------------

if __name__ == '__main__':
    # 取当前目录的Raw目录生成路径
    TopPath = BaseDir.get_upper_dir(curPath, -1)
    sPath = BaseDir.get_new_path(TopPath, "html")
    print('path:' + sPath)
    # 遍历子目录
    for filePath in BaseDir.get_dir_all_files(sPath):
        print(filePath)
        GetFileName(filePath)

        if len(ListSqls) > 0:
            InsertIntoDbFromList()

        print('DOWN TOTAL:' + str(nCount))
